package com.job.process;

import com.job.pojo.Java;
import com.job.utils.MyUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.List;


@Component
public class MyProcessor implements PageProcessor {
    //最初访问的网址，静态注入
    private static String url;
    @Value("${web.url}")
    public void setUrl(String url){
        MyProcessor.url=url;
    }
    //存放图片文件的路径
    private static String savePicPath;
    @Value("${pic.path:F:\\java\\testClimp}")
    public void setSavePicPath(String savePicPath){
        MyProcessor.savePicPath=savePicPath;
    }
    private final Site site=Site.me()
            .setRetryTimes(3)//重试5次
            .setRetrySleepTime(5000)//5s间隔
            .setTimeOut(10*1000) //连接超时
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0")
            .setCharset("utf-8");//10s请求时间
    private final MyUtils myUtils =new MyUtils();
    //获取工作的时间
    public static String jobTime;
    @Override
    @Transactional
    public void process(Page page) {
        //获取招聘信息页面
        Html html=page.getHtml();
        //有列表代表是列表页，否则是详情页
        Selectable selectable = html.$("#list_con li[class=job_item clearfix]");
        if (selectable!=null&&selectable.all().size()!=0){//列表页
            //取出链表,可以遍历
            List<Selectable> nodes = selectable.nodes();
            //当进入数据库时，如果数据库当前有内容，则清空
            if (myUtils.getNums()!=0){
                myUtils.delData();
            }
            for (Selectable element:nodes){
                String url=element.$("a").links().get();
                url=url.substring(0,url.indexOf("?"));//太长了需要截取一部分,否则不能插入数据库 链接
                String place=element.xpath("//span[@class='address']/text()").toString();//工作地点
                String jobName=element.xpath("//span[@class='name']/text()").toString(); //工作名称
                String salary=element.xpath("//p[@class='job_salary']/text()").toString()//薪资
                        +element.xpath("//p[@class='job_salary']/i[@class='unit']/text()")
                        .toString();
                String good=element.xpath("//div[@class='job_wel clearfix']/span/text()").get();//福利
                String companyName=element.$(".fl","text").get(); //公司名称
                String pushTime=element.$(".sign","text").get();//发布时间
                //给对象赋值
                Java java=new Java(null,jobName,pushTime,salary,place,good,companyName,url);
                //进行数据库操作
                myUtils.caoZuo(java);
                //显示本次插入对象信息
//                System.out.print("链接:"+url);
//                System.out.print(",地点:"+place);
//                System.out.print(",工作名称:"+jobName);
//                System.out.print(",薪资:"+salary);
//                System.out.print(",福利:"+good);
//                System.out.print(",公司名称:"+companyName);
//                System.out.print(",发布时间:"+pushTime);
//                System.out.println();
            }
            System.out.println("成功加载java工作数据");
        }else if ((html.$("title","text").toString())==null){
            List<String> picList=new ArrayList<>();//图片List
            picList.add(url);//放入图片的url
            picList.add(url);
            page.putField(ImagePipeline.urlKey,picList);
            System.out.println("错误页面");
        }else if ("下载图片".equals(html.get())){//下载图片
            List<String> picList=new ArrayList<>();//图片List
            picList.add(url);//放入图片的url
            picList.add(url);
            page.putField(ImagePipeline.urlKey,picList);
        }
        else if (html.$("title","text").toString().contains("请输入验证码")){//详情页
            System.out.println("遭遇浏览器拦截");
        }else {
            System.out.println("进入到详情页");
//            page.addTargetRequests();//如果要继续获取信息，将上面链接的url存入List集合放入括号即可，再填写要捕捉的信息即可
        }
    }

    @Override
    public Site getSite() {
        return site;
    }
    //initialDelay启动话间隔多久执行，fixedDelay每次之间间隔多久
    @Scheduled(initialDelay = 1000,fixedDelay = 1000*60*60*24*7)
    public void startClimp(){
//        //创建下载器
//        HttpClientDownloader downloader=new HttpClientDownloader();
//        //给下载器配置代理服务器信息
//        downloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("111.225.153.105",3256)));
//        启动
        Spider spider = Spider.create(new MyProcessor())
//                .thread(5)//多少个线程
                .addUrl(url)//访问的网址
//                .setDownloader(downloader) //设置下载器
                .addPipeline(new ImagePipeline(savePicPath))//下载图片
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)));//bloom去重
        spider.run();
        //发送本次的工作获取时间
        jobTime=myUtils.getJobTime();
    }
}
